{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This notebook has been tested with \n",
    "* Python 3.5\n",
    "* Keras 2.0.8\n",
    "* Tensorflow 1.3.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.layers import Input, Flatten, Dense\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use VGG16 model with pre-trained weights\n",
    "* Keras documentation for detail: \n",
    "https://keras.io/applications/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,357,544\n",
      "Trainable params: 138,357,544\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Use whole vgg16 model\n",
    "# Input image format: (224 X 224 X 3)\n",
    "vgg16_with_top = VGG16(include_top=True, weights='imagenet',\n",
    "                                input_tensor=None, input_shape=None,\n",
    "                                pooling=None,\n",
    "                                classes=1000)\n",
    "vgg16_with_top.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use only convolutional part of VGG16 and add dense layer\n",
    "* Create your own input format\n",
    "* Create your own model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, None, None, 3)     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, None, None, 64)    1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, None, None, 64)    36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, None, None, 64)    0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, None, None, 128)   73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, None, None, 128)   147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, None, None, 128)   0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, None, None, 256)   295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, None, None, 256)   0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, None, None, 512)   1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 14,714,688\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "image_input (InputLayer)     (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "vgg16 (Model)                multiple                  14714688  \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              25089000  \n",
      "=================================================================\n",
      "Total params: 39,803,688\n",
      "Trainable params: 25,089,000\n",
      "Non-trainable params: 14,714,688\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:22: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"im..., outputs=Tensor(\"pr...)`\n"
     ]
    }
   ],
   "source": [
    "# Get back the convolutional part of a VGG network trained on ImageNet\n",
    "model_vgg16_conv = VGG16(weights='imagenet', include_top=False)\n",
    "model_vgg16_conv.summary()\n",
    "\n",
    "# Stop to train weights of convolutional layers, if you want to fit them, markdown the following two lines\n",
    "for layer in model_vgg16_conv.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# Create your own input format (here 224 X 224 X 3)\n",
    "inputs = Input(shape=(224,224,3),name = 'image_input')\n",
    "\n",
    "# Use the generated model \n",
    "output_vgg16_conv = model_vgg16_conv(inputs)\n",
    "\n",
    "# Add the fully-connected layers \n",
    "x = Flatten(name='flatten')(output_vgg16_conv)\n",
    "#x = Dense(128, activation='relu', name='fc1')(x)\n",
    "#x = Dense(128, activation='relu', name='fc2')(x)\n",
    "x = Dense(1000, activation='softmax', name='predictions')(x)\n",
    "\n",
    "# Create your own model \n",
    "my_model = Model(inputs=inputs, output=x)\n",
    "\n",
    "# In the summary, weights and layers from VGG part will be hidden, and they will not be fit during traning\n",
    "my_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine tune VGG16 model with a designated layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              25089000  \n",
      "=================================================================\n",
      "Total params: 39,803,688\n",
      "Trainable params: 25,089,000\n",
      "Non-trainable params: 14,714,688\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:18: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"pr...)`\n"
     ]
    }
   ],
   "source": [
    "# Generate a model with all layers (with top)\n",
    "my_vgg16_model = VGG16(weights='imagenet', include_top=True)\n",
    "\n",
    "# Stop to train weights of VGG16 layers\n",
    "for layer in my_vgg16_model.layers:\n",
    "    layer.trainable = False\n",
    "    \n",
    "# Add a layer where input is the output of the second last layer\n",
    "#x = Flatten(name='flatten')(my_vgg16_model.layers[-2].output)\n",
    "\n",
    "# Add a layer where input is the output of the fourth last layer\n",
    "x = Dropout(0.9, noise_shape=None, seed=None)(my_vgg16_model.layers[-4].output) \n",
    "#x = Dense(128, activation='relu', name='fc1')(x)\n",
    "#x = Dense(128, activation='relu', name='fc2')(x)\n",
    "x = Dense(1000, activation='softmax', name='predictions')(x)\n",
    "\n",
    "# Then create the corresponding model \n",
    "my_model = Model(inputs=my_vgg16_model.input, output=x)\n",
    "my_model.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
